package com.bedpotato.thread;

import java.util.HashSet;
import java.util.Set;

import javax.annotation.Resource;

import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;

import com.bedpotato.constants.ProxyClawerConstants;
import com.bedpotato.http.HtmlUnitMethod;
import com.bedpotato.po.ProxyPO;
import com.bedpotato.service.ProxyClawerService;
import com.bedpotato.util.ProxyUtil;

@Component
public class FreeproxyListsSiteClawerThread implements Runnable {
	Logger logger = Logger.getLogger(FreeproxyListsSiteClawerThread.class);
	@Resource
	ProxyClawerService proxyClawerService;
	public static void main(String[] args) {
		FreeproxyListsSiteClawerThread freeproxyListsSiteClawerThread = ProxyClawerConstants.applicationContext.getBean("freeproxyListsSiteClawerThread",FreeproxyListsSiteClawerThread.class);
		freeproxyListsSiteClawerThread.run();
	}
	@Override
	public void run() {
		logger.info("FreeproxyLists信息抓取启动");
		Set<ProxyPO> proxySet = getFreeproxylitsData();
		proxyClawerService.saveProxy(proxySet);
		logger.info("FreeproxyLists信息抓取结束");
	}

	public Set<ProxyPO> getFreeproxylitsData() {
		HtmlUnitMethod htmlUnitMethod = new HtmlUnitMethod();
		Set<ProxyPO> proxySet = new HashSet<ProxyPO>();
		for (int i = 1; i <= 8; i++) {
			try {
				String pageStr = htmlUnitMethod
						.get("http://www.freeproxylists.net/zh/?c=CN&u=70&page=" + i);
				String keywordPageStr = StringUtils.substringBetween(pageStr,
						"12345", "12345").trim();
				String[] contentStrArray = keywordPageStr.split("\n");
				for (String contentStr : contentStrArray) {
					try {
						if (!contentStr.trim().isEmpty()
								&& ProxyUtil
										.isLegalIP(contentStr.trim().substring(
												0, contentStr.indexOf("\t")))) {
							ProxyPO proxyPO = new ProxyPO();
							proxyPO.setIp(contentStr.trim().substring(0,
									contentStr.indexOf("\t")));
							proxyPO.setPort(contentStr
									.trim()
									.substring(contentStr.indexOf("\t"),
											contentStr.indexOf("H")).trim());
							// System.out.println(proxyPO.getIp() + ":"
							// + proxyPO.getPort());
							proxySet.add(proxyPO);
						}
					} catch (Exception e) {
						// 正常
					}
				}
			} catch (Exception e) {
				logger.error("FreeproxyLists " + e);
			}
		}
		return proxySet;
	}
}
